from scrapy.selector import Selector
from spidertools.utils.xpath_utils import get_alltext,get_all_tables
from spidertools.utils.text_utils import replace_punctuation
from spidertools.utils.snippets import combine_two_dict
from info_fsm import InfoMachine
from pprint import pprint
import re
import os
from table_info_extract.extract_utils import table_info_extract_styletwo,table_info_extract_stylethree
from table_info_extract import dict_mapping_triedTree
from table_info_extract.common_table_utils import common_table_extrator
from extractors.base_extractor import BaseExtractor
from table_info_extract.table_fsm import TableExtractMachine
from utils.logger_utils import savelog, LogStateString



class JiangSuJianSheGongChengZhaoBiaoWang(BaseExtractor):
    def __init__(self,info_dict,):
        super(JiangSuJianSheGongChengZhaoBiaoWang, self).__init__(info_dict)


    def parse_bid_evaluation_results_announcement(self):
        '''
        评标结果公告提取
        :return:
        '''
        body_text = get_alltext(self.sel.xpath('//body'))
        machine = InfoMachine(self.base_pattern,extend_keywords=self.extend_keywords)
        clean_text  = replace_punctuation(body_text.strip())

        output = machine.run_list([clean_text])
        #output_dict = combine_two_dict(output, self.info_dict)

        tables = get_all_tables(self.sel)
        for table in tables:
            result = common_table_extrator(table)
            if result:
                output = combine_two_dict(output, result)

        zhaobiao_own = self.sel.xpath("//*[@id='lblJsDW']/text()")
        if zhaobiao_own:
            ower = zhaobiao_own.extract()[0]
            if ower:
                if '工程招标信息' not in output:
                    output['工程招标信息'] = {}

                if '招标人信息名称' not in output['工程招标信息']:
                    output['工程招标信息']['招标人信息名称'] = []
                if ower not in output['工程招标信息']['招标人信息名称']:
                    output['工程招标信息']['招标人信息名称'].append(ower)

        xiangmu_name = self.sel.xpath("//*[@id='lblBDName']/text()")
        if xiangmu_name:
            name = xiangmu_name.extract()[0]
            if '工程招标信息' not in output:
                output['工程招标信息'] = {}

            if '标段名称' not in output['工程招标信息']:
                output['工程招标信息']['标段名称'] = name
        return output


    def parse_not_shortlisted_announcement(self):
        '''
                评标未入围公告信息提取
                :return:
                '''
        output_dict = {}
        tables = get_all_tables(self.sel)
        for table in tables:
            result = common_table_extrator(table)
            output_dict = combine_two_dict(output_dict,result)


        return output_dict

    def commom_announcement_parse(self):
        '''
        通用网页公告解析
        :return:
        '''

        # 获取招标公告正文
        content_root_nodes = self.sel.xpath('//body')#调用了之前设置的selector对象
        texts = []

        # 遍历所有子节点，获取相应的文本
        for node in content_root_nodes:
            node_text = get_alltext(node)
            texts.append(node_text)

        clean_texts = []
        # 清理文本中的一些空几个，不可见字符，以及加载关键字中间的空格
        for text in texts:
            text = replace_punctuation(text.strip())#strip用于去除首尾空格,replace_punctuation去除各种符号
            clean_texts.append(text)

        # 创建状态机
        machine = InfoMachine(self.base_pattern,extend_keywords=self.extend_keywords)#调用了info_fsm.py的InfoMachine,base_pattern = '%s:\s*(.*)'
        # 执行状态机，解析整个文本
        output_dict = machine.run_list(clean_texts)#调用实例化后的run_list方法



        #获取所有的table表格
        tables = get_all_tables(self.sel)#该方法与get_alltext一样来自于spidertools工具中
        for table in tables:
            result = common_table_extrator(table)#调用通用表格转换模板
            output_dict = combine_two_dict(output_dict, result)

        return output_dict


    def parse_win_bidding_announcement(self):
        '''
        中标公告信息提取
        :return:
        '''
        table_extractor = TableExtractMachine()
        table_node = self.sel.xpath('//table[@id="Table1"]')
        output_dict = table_extractor.run_table(table_node)
        return output_dict


    def parse_bid_announcemen(self):
        '''
        招标信息提取
        :param root:
        :return:
        '''
        self.html = replace(self.html)
        self.sel = Selector(text=self.html)
        output_dict = {}
        # 获取所有的table表格
        # self.table_extract_machine = TableExtractMachine(one_line_mode_return_result=False)
        tables = self.sel.xpath('//table')
        tables = table_first_choice(tables)
        first=first_table(tables[0])
        tables[0] = first
        tables = table_second_choice(tables)
        for table in tables:
            result = self.table_extract_machine.run_table(table)
            output_dict = combine_two_dict(output_dict, result)
        info_dict = output_dict


        #获取招标公告正文
        # content_root_nodes = self.sel.xpath('//tr[@id="trzygg"]')
        content_root_nodes = self.sel.xpath('//tr')
        if content_root_nodes:
            texts = []
            texts_two = []
            # 遍历所有子节点，获取相应的文本
            for node in content_root_nodes:
                node_text = get_alltext(node)
                if '特殊公告'  in node_text:
                    pass
                elif '工程招标文件备案表' in node_text:
                    pass
                else:
                    if "投标报名联系人及联系方式" in node_text:
                        if "投标报名联系人及联系方式:" in node_text:
                            pass
                        elif "投标报名联系人及联系方式:" not in node_text :
                            node_text = node_text.replace('投标报名联系人及联系方式', '投标报名联系人及联系方式:')
                    else:
                        pass
                    texts.append(node_text)

            # 清理文本中的一些空几个，不可见字符，以及加载关键字中间的空格
            for text in texts:
                text = replace_punctuation(text.strip())
                text = text.replace('\n', '')
                text = text.replace('\r', '')
                text = text.replace('\xa0', '')
                text = text.replace('\t', '')
                text = text.replace('\u3000', '')
                texts_two.append(text)
            # 执行状态机，解析整个文本
            # output = {}
            output = self.texts_matchine_parse(texts_two)#文本解析
            output_dict = combine_two_dict(output, info_dict)

        try:
            if '标段名称' not in output_dict['工程招标信息'].keys() or output_dict['工程招标信息']['标段名称']=='':
                output_dict['工程招标信息']['标段名称'] = find_project_name(output_dict['工程基本信息']['项目名称'])
        except Exception as q:
            pass
        output_dict = clean_dict(output_dict)  # 消除多余字符并且去重
        output_dict = re_juhao(output_dict)
        return output_dict

    def check_purchase(self):#不解析所有采购公告
        return_flag = False
        if '采购' in self.info_dict['公告标题']:
            return_flag = True
        return return_flag

    def start_parse(self):
        result = {}
        if self.check_purchase():
            savelog(LogStateString.Waring_NoNeedParse, msg_dict={"url": self.info_dict["origin_url"]})
            return {"not_need_checkjson": '1'}
        if '招标公告' in self.info_dict['公告类型']:
            result = self.parse_bid_announcemen()
        elif '中标公告' in self.info_dict['公告类型']:
            result = self.parse_win_bidding_announcement()
        elif '未入围公示' in self.info_dict['公告类型']:
            pass
            #result = self.parse_not_shortlisted_announcement()
        elif '评标结果公示' in self.info_dict['公告类型']:
            pass
            #result = self.parse_bid_evaluation_results_announcement()

        return result





def replace(html):
    html = html.replace('：', ':')
    html = html.replace('（', '(')
    html = html.replace('）', ')')
    clean_list = ['请投标人登陆','本招标工程共分','投标保证金必须','本公告共划分成','中标候选人','本项目执行','负责人(电子签名)']#去除不需要的后缀
    for i in clean_list:
        if i in html:
            html = html.replace(i , '不要:')

    if '(网址' or '网址:'  in html:
        html = html.replace('(网址', '不要:')
        html = html.replace('网址:', '不要:')

    zhu_list =['注','注1','注2','注3']
    for zhu in zhu_list:
        a= zhu+':'
        if a in html:
            html = html.replace(a, '不要:')

    if '重要提醒:' not in html and '重要提醒' in html:
        html = html.replace('重要提醒', '重要提醒:')
    if '注意事项:' not in html and '注意事项' in html:
        html = html.replace('注意事项', '注意事项:')
    if '资格审查内容:' not in html and '资格审查内容' in html:
        html = html.replace('资格审查内容', '资格审查内容:')


    no_need = ['网址', '联系人', '地址', '传真', '联系电话', '电话', '邮箱']

    for i in no_need:
        a = '(' + i
        if a  in html:
            html = html.replace(a, '不要')


    a_list = ['邮箱', '联系方式', '联系人','网址', '地址', '传真', '联系电话', '电话','地点']
    first_list = [';', '，', '。']
    second_list = [':', '：']

    for a in a_list:
        for first in first_list:
            for second in second_list:
                nono = first + a + second
                if nono in html:
                    html = html.replace(nono, '不要:')

    if  '(投)' in html:
        html = html.replace('(投)', '投')
        if '招投标文件制作工具使用说明:'  not in html and '招投标文件制作工具使用说明' in html:
            html = html.replace('招投标文件制作工具使用说明', '招投标文件制作工具使用说明:')

    if '开标当日' in html:
        html = html.replace('开标当日', '开标当日:')
    if  '投标报价:' or '工程总承包报价' or '投标报价总分:' or '投标报价('  in html:
        html = html.replace('投标报价:', '不要:')
        html = html.replace('工程总承包报价', '不要')
        html = html.replace('投标报价总分:', '不要:')
        html = html.replace('投标报价(', '不要:')

    if '（资格后审） 标段编号:' or '公告 标段编号:' or '（资格后审） 标段编号:' in html:
        html = html.replace('（资格后审） 标段编号:', '不要:')
        html = html.replace('公告 标段编号:', '不要:')
    if '（资格后审） 项目编号:' or '公告 项目编号:' in html:
        html = html.replace(' （资格后审） 项目编号:', '不要:')
        html = html.replace('公告 项目编号:', '不要:')
    if '一标段:' or '二标段:'  in html:
        html = html.replace('一标段:', '')
        html = html.replace('二标段:', '')

    if '请投标人登陆' or '(本次招标的' in html:
        html = html.replace('请投标人登陆', '请投标人登陆:')
        html = html.replace('(本次招标的', '请投标人登陆:')

    return html

def table_first_choice(tables):
    new_tables = []
    for table in tables:
        tr = table.xpath('./tr')
        tab = table.xpath('//table')
        if len(tr) == 1:
            continue
        elif len(tab) >= 1 and not tab:
            continue
        else:
            new_tables.append(table)
    return new_tables
def table_second_choice(tables):
    new_tables = []
    no_need_word=['初步评审','评分内容','评分标准','评审内容','CA名称','软件公司名称','CA签章办理指南','招投标行政监督部门','评审因素',
                  '评审标准']
    for table in tables:
        x=[]
        text = table.xpath('string(.)').extract()[0]
        for i in no_need_word:
            if i in text:
                x.append(i)
        if not x:
            new_tables.append(table)
    return new_tables

def first_table(table):
    tr_list=table.xpath('tr')
    new_tr_list =[]
    for tr in tr_list:
        tr_text = tr.getall()[0]
        if 'DISPLAY: none' in tr_text:
            continue
        else:
            new_tr_list.append(tr.getall()[0])
    new_tr_list = new_tr_list[0:6]
    new_table = ''.join('%s' % id for id in new_tr_list)  # 因为有数字，所以要用这个
    new_table = '<table>' + new_table + '</table>'
    table = Selector(text=new_table)
    table_node = table.xpath('//table')[0]
    return table_node


def clean_dict(output_dict):   #提供更干净的输出内容
    try:
        for x in output_dict["工程基本信息"].keys():
            if type(output_dict["工程基本信息"][x])==str:
                output_dict["工程基本信息"][x] = output_dict["工程基本信息"][x].replace(' ', '')
                output_dict["工程基本信息"][x] = output_dict["工程基本信息"][x].replace('\xa0', '')
                output_dict["工程基本信息"][x] = output_dict["工程基本信息"][x].replace('\t', '')
            else:
                pass
    except Exception as a:
        pass
    try:
        for i in output_dict["工程招标信息"].keys():
            if type(output_dict["工程招标信息"][i]) == list:
                new_list =[]
                for a in output_dict["工程招标信息"][i]:
                    a = a.replace(' ', '')
                    a = a.replace('\xa0', '')
                    a = a.replace('\t', '')
                    new_list.append(a)
                output_dict["工程招标信息"][i] = new_list#消除多余字符
                output_dict["工程招标信息"][i] = list(set(output_dict["工程招标信息"][i]))#去重
            else:
                output_dict["工程招标信息"][i] = output_dict["工程招标信息"][i].replace('\xa0', '')
                output_dict["工程招标信息"][i] = output_dict["工程招标信息"][i].replace('\t', '')
                pass
        # if output_dict['未知']['未知邮编']==['E－mail']:
        #     del output_dict['未知']['未知邮编']
        # return output_dict
    except Exception as o:
        pass
    try:
        if output_dict['未知']['未知邮编']==['E－mail']:
            del output_dict['未知']['未知邮编']
    except Exception as n:
        pass
    try:
        if len(output_dict["工程招标信息"]['招标代理机构邮箱'])>1:#处理多匹配了的邮箱
            x=[]
            for i in output_dict["工程招标信息"]['招标代理机构邮箱']:
                if '年' or '\r' in i :
                    pass
                else:
                    x.append(i)
            output_dict["工程招标信息"]['招标代理机构邮箱'] = x
    except Exception as a:
        pass
    return output_dict

def table_change(table):
    tr_list = table.xpath('tr')
    new_tr_list =[]
    for tr in tr_list:
        if 'trzygg' in tr.getall():
            pass
        else:
            new_tr_list.append(tr.getall())
    x= ''.join(new_tr_list)
    table = '<table>' + x + '</table>'
    return table

def re_juhao(output_dict):
    try:
        if '。' in output_dict["工程基本信息"]["工程基本信息工期"] and len(output_dict["工程基本信息"]["工程基本信息工期"])>100:
            output_dict["工程基本信息"]["工程基本信息工期"] = re.findall('(.*?)。',output_dict["工程基本信息"]["工程基本信息工期"])[0]
    except Exception as a:
        pass
    try:
        if '。' in output_dict["工程招标信息"]["招标人信息工期"] and len(output_dict["工程招标信息"]["招标人信息工期"])>100:
            output_dict["工程招标信息"]["招标人信息工期"] = re.findall('(.*?)。',output_dict["工程招标信息"]["招标人信息工期"])[0]
    except Exception as a:
        pass
    try:
        if '。' in output_dict["工程基本信息"]["建设周期"] and len(output_dict["工程基本信息"]["建设周期"])>100:
            output_dict["工程基本信息"]["建设周期"] = re.findall('(.*?)。',output_dict["工程基本信息"]["建设周期"])[0]
    except Exception as a:
        pass
    try:
        if '。' in output_dict["工程招标信息"]["标段工期"] and len(output_dict["工程招标信息"]["标段工期"])>100:
            output_dict["工程招标信息"]["标段工期"] = re.findall('(.*?)。',output_dict["工程招标信息"]["标段工期"])[0]
    except Exception as a:
        pass
    try:
        if '。' in output_dict["工程招标信息"]["投标截止时间"] and len(output_dict["工程招标信息"]["投标截止时间"])>100:
            output_dict["工程招标信息"]["投标截止时间"] = re.findall('(.*?)。',output_dict["工程招标信息"]["投标截止时间"])[0]
    except Exception as a:
        pass
    return output_dict

def find_project_name(gonggao_name):  # 在取得项目名称时，遇如下非工程关键字，则保留项目完整名称

    if '招标公告' in gonggao_name:
        if '项目' in gonggao_name and '工程' in gonggao_name:
            project_name = re.findall("(.*?)项目", gonggao_name)[0] + '项目'
        elif '项目' in gonggao_name and '工程' not in gonggao_name:
            project_name = re.findall("(.*?)项目", gonggao_name)[0] + '项目'
        elif '项目' not in gonggao_name and '工程' in gonggao_name:
            project_name = re.findall("(.*?)工程", gonggao_name)[0] + '工程'
        else:
            project_name = gonggao_name.replace('招标公告', '')
    elif '招标公告' not in gonggao_name:
        if '项目' in gonggao_name and '工程' in gonggao_name:
            project_name = re.findall("(.*?)项目", gonggao_name)[0] + '项目'
        elif '项目' in gonggao_name and '工程' not in gonggao_name:
            project_name = re.findall("(.*?)项目", gonggao_name)[0] + '项目'
        elif '项目' not in gonggao_name and '工程' in gonggao_name:
            project_name = re.findall("(.*?)工程", gonggao_name)[0] + '工程'
        else:
            project_name = gonggao_name
    return project_name



if __name__ == '__main__':
    info_dict = {}
    path ='../../../demo_html/sun'
    for root, dirs, files, in os.walk(path):
        for i,name in enumerate(files):
            if "init" not in name:
                print(f"\033[32m{i+1}=============={name}\033[0m")
                # with open(f'../../../demo_html/zhaobiao/{name}', 'r', encoding='utf-8') as fr:
                with open(f'{path}'+'/'+f'{name}', 'r', encoding='utf-8') as fr:
                    html = fr.read()
                info_dict['html'] = html
                info_dict['公告类型'] = "招标公告"  # "中标结果公告" info_dict['公告类型'] = "中标公告"  # "中标结果公告"
                info_dict["source_type"] = "江苏建设工程招标网"
                info_dict['province'] ="江苏"
                info_dict['_id'] = "1"
                info_dict["origin_url"] = "1"
                info_dict['公告标题'] = 'jjp'
                obj = JiangSuJianSheGongChengZhaoBiaoWang(info_dict)
                result = obj.start_parse()
                pprint(result)


#
# if __name__ == '__main__':
#     import requests
#     zhaobiao_url = "http://www.jszb.com.cn/jszb/YW_info/ZhaoBiaoGG/ViewReportDetail.aspx?RowID=692652&categoryNum=012&siteid=1"
#     zhongbiao_url = "http://www.jszb.com.cn/JSZB/yw_info/ZhongBiaoGS/ViewGSDetail.aspx?RowID=1297866&siteid=1&categoryNum=7"
#     not_in_url = "http://www.jszb.com.cn/JSZB/yw_info/ZiGeYS/ViewYSDetail.aspx?RowID=158068&siteid=1&categoryNum=8"
#     bid_evaulate_url = "http://www.jszb.com.cn/JSZB/yw_info/HouXuanRenGS/ViewHXRDetail.aspx?GongGaoGuid=ec62fb01-6c99-4863-aecc-6eb081380a30"
#
#     req = requests.get(zhaobiao_url)#通过修改网址，得到不同的结果
#     info_dict = {}
#     info_dict['html'] = req.text
#     info_dict['公告类型'] = "招标公告"
#     obj = JiangSuJianSheGongChengZhaoBiaoWang(info_dict)#把信息传给类
#     result = obj.output_extractor_dict()#解析
#     pprint(result)